#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Project: spd-sxmcc
"""
@author: lyndon
@time Created on 2018/11/26 12:51
@desc
"""
import re
import time
import urllib2, random, requests
from pyquery import PyQuery as pyq

from ippool.lib.pooldao.ip_pool_dao import IpPool

t = time.time()
nowTime = lambda: int(round(t * 1000))

httphead = '''
Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Encoding:gzip, deflate
Accept-Language:zh-CN,zh;q=0.8
Cache-Control:max-age=0
Connection:keep-alive
Cookie:webim_token_5452353=YWMtYskIqPLSEeigMI1J4UU2D0hiN_DRZBHmt4eJit69xiBgGzWQ8tIR6JLHeWnuG_HFAwMAAAFnWOQg9wBPGgAXPS6j2wIpMdFjg43iGPQcDxczlDMktcDy48CKi7ZyZQ; _Jo0OQK=37A7F8C916E708499CA68AF1BDA22504728EC18F01A1ED4F536E61E79A0C0077D121CC549C3FCB9DFDC13E3C4DEED15B5A7F2DDD81879C25F3122CE4F51BF72391BB8270981362057E932F58BDF95BB9E2932F58BDF95BB9E297D346AF2564C2E6FGJ1Z1Vw==; _ga=GA1.2.1033270354.1543306173; _gid=GA1.2.1680794169.1543306173; yfx_c_g_u_id_10000001=_ck18112716093311658135487817974; yfx_mr_n_10000001=baidu%3A%3Amarket_type_ppzq%3A%3A%3A%3A%3A%3A%3A%3A%3A%3A%25E6%25A0%2587%25E9%25A2%2598%3A%3Aty.5i5j.com%3A%3A%3A%3A%3A%3A%25E5%25B7%25A6%25E4%25BE%25A7%25E6%25A0%2587%25E9%25A2%2598%3A%3A%25E6%25A0%2587%25E9%25A2%2598%3A%3A189%3A%3Apmf_from_adv%3A%3Aty.5i5j.com%2F; yfx_mr_f_n_10000001=baidu%3A%3Amarket_type_ppzq%3A%3A%3A%3A%3A%3A%3A%3A%3A%3A%25E6%25A0%2587%25E9%25A2%2598%3A%3Aty.5i5j.com%3A%3A%3A%3A%3A%3A%25E5%25B7%25A6%25E4%25BE%25A7%25E6%25A0%2587%25E9%25A2%2598%3A%3A%25E6%25A0%2587%25E9%25A2%2598%3A%3A189%3A%3Apmf_from_adv%3A%3Aty.5i5j.com%2F; yfx_key_10000001=; ershoufang_cookiekey=%5B%22%257B%2522url%2522%253A%2522%252Fershoufang%252F_%2525E4%2525B8%252589%2525E8%2525A7%252592%2525E6%2525B4%2525B2%253Fzn%253D%2525E4%2525B8%252589%2525E8%2525A7%252592%2525E6%2525B4%2525B2%2522%252C%2522x%2522%253A%25220%2522%252C%2522y%2522%253A%25220%2522%252C%2522name%2522%253A%2522%25E4%25B8%2589%25E8%25A7%2592%25E6%25B4%25B2%2522%252C%2522total%2522%253A%25220%2522%257D%22%5D; ershoufang_BROWSES=41895348%2C35419242; Hm_lvt_94ed3d23572054a86ed341d64b267ec6=1543306174,1543376654; xiaoqu_BROWSES=358503%2C368303%2C368031; yfx_f_l_v_t_10000001=f_t_1543306173158__r_t_1543376653460__v_t_1543384366073__r_c_1; yfx_s_u_id_10000001=5452353; yfx_s_u_name_10000001=17696030367; domain=ty; PHPSESSID=ST-140863-FgDJAd4sRZOiKbbvwHPZ-passport5i5jcom; user_info=TBYRRFZKXlNdAUFbEANWAQoHVQAAAwMPQ08VXF1RDgtRXVUSChJKSQIBCQgBAhIcEkVDVUJ5VBIKEgUEBQIDBQMSTQ; wiwj_token_ticket=ST-140863-FgDJAd4sRZOiKbbvwHPZ-passport.5i5j.com; wiwj_token_ST-140863-FgDJAd4sRZOiKbbvwHPZ-passport.5i5j.com=%7B%22uid%22%3A%225452353%22%7D; _gat=1; Hm_lpvt_94ed3d23572054a86ed341d64b267ec6=1543384604
Host:bj.5i5j.com
Referer:http://bj.5i5j.com/rent
Upgrade-Insecure-Requests:1
User-Agent:Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36
'''


def init():
    headers = ("User-Agent",
               "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0")
    opener = urllib2.build_opener()
    opener.addheaders = [headers]

    urllib2.install_opener(opener)
    data = urllib2.urlopen(
        url="http://www.thebigproxylist.com/members/proxy-api.php?output=all&user=list&pass=8a544b2637e7a45d1536e34680e11adf"
    ).read().decode('utf8')
    ippool = data.split('\n')

    total_rows = len(ippool)
    print(len(ippool))

    try:
        for ip in ippool:
            if ip.strip() != '':
                ip = ip.decode('utf-8').encode('ascii', 'ignore')
                print(ip)
                ip_port, protocol, get_time = ip.split(',')
                ippoolobj = IpPool(ip_port=ip_port, protocol=protocol, get_time=get_time)
                ippoolobj.insert_one()
    except Exception as e:
        print('Executor Error', e)
    else:
        print('Insert ', total_rows, 'rows')


def getAll():
    return IpPool().select()


def get_random_ip():
    proxy_ip_pools = getAll()
    random_proxy_ip = random.choice(proxy_ip_pools)
    return random_proxy_ip.ip_port


def get_head_cookies(httphead):
    head = {}
    for i in httphead.strip().split('\n'):
        if re.match('Referer', i):
            head['Referer'] = i[len('Referer'):]
            continue
        line = i.strip().split(':')
        head[line[0]] = line[1]
    cookie = head['Cookie']
    cookies = {}
    for i in cookie.strip().split(';'):
        line = i.strip().split('=')
        cookies[line[0]] = line[1]
    return head, cookies


if __name__ == '__main__':
    # init()

    # hs = HeadersSelector()
    # header = hs.select_header()

    header, cookies = get_head_cookies(httphead)
    print(header)
    print(cookies)
    print(cookies['Hm_lpvt_94ed3d23572054a86ed341d64b267ec6'])
    tt = nowTime() / 1000
    print(tt)
    cookies['Hm_lpvt_94ed3d23572054a86ed341d64b267ec6'] = str(tt)
    print(cookies)
    print(cookies['Hm_lpvt_94ed3d23572054a86ed341d64b267ec6'])
    random_proxy_ip = get_random_ip()
    print('proxy', random_proxy_ip)
    print(type(random_proxy_ip))
    s = requests.session()

    r = s.get('https://ty.5i5j.com/xiaoqu/358602.html', proxies={"http": random_proxy_ip}, headers=header,
              cookies=cookies)
    # r = requests.get('http://www.baidu.com', proxies={"http": '121.33.220.158:808'})
    r.encoding = 'utf-8'
    # r.encoding = r.apparent_encoding
    print(type(r))
    html = r.text
    print(html)
    print(type(html))
    py_html = pyq(html, parser="html")
    print(py_html)
    print(type(py_html))

    print('---------------------------------------------------------')
    print(r.cookies.get_dict())
    title = py_html('.house-tit').text()
    price = py_html('.xqjprice > span').text()
    print(title)
    print(price)
    print('---------------------------------------------------------')
    for each in py_html('.xqfangs ul li span').items():
        print(each)
        print(str(each).encode('utf-8'))
        print(type(each))
